import requests
import json, re
from dbutil.pgsql import PgsqlPipeline
from datetime import date
from lxml import etree


def get_content():
    headers = {

        'apiKey': 'JAdOBlxEg2wRRQKJus8D9R5kjvDZ78Ba',
        'clientKey': 'lVqTrQx76FnGUhV6AFi7iSy9aXRwLIy7',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36'
    }
    url = "https://ap.nissan-api.net/v2/dealers?size=200&lat=39.91092462465681&long=116.4133836971231&serviceFilterType=AND&include=openingHours"

    response = requests.get(url, headers=headers)
    dealer = json.loads(response.text)
    return dealer


def get_item(dealer):
    item = {
        "dealer_name": dealer["name"],
        "brand_id": None,
        "address": dealer["address"]["addressLine1"],
        "brand": "英菲尼迪",
        "province":dealer["address"]["state"] ,
        "city": dealer["address"]["city"],
        "sale_call": None,
        "customer_service_call": dealer["contact"]["phone"],
        "update_time": date.today(),
        "longitude": dealer['geolocation']['longitude'],
        "latitude": dealer['geolocation']['latitude'],
        "dealer_type": None,
        "manufacturer_id": None,
        "manufacturer": "英菲尼迪",
        "state": None,
        "opening_date": None,
        "close_date": None,
        "dealer_id_web": None,
        "controlling_shareholder": None,
        "other_shareholders": None,
        "status": None,
        "remarks": None,

    }

    return item


def get_items():
    items = []
    dealers = get_content()
    for dealer in dealers["dealers"]:
        item = get_item(dealer)
        items.append(item)
    return items


def main():
    print('爬虫开始.....')
    items = get_items()

    print('等待数据存储')
    pg = PgsqlPipeline()
    for item in items:
        pg.process_item(item)
    pg.close()
    print('数据存储完成！！！')


if __name__ == '__main__':
    main()
